import scrapy
from selenium import webdriver
from wangyiPro.items import WangyiproItem

class WangyiSpider(scrapy.Spider):
    name = 'wangyi'
    # allowed_domains = ['ww']
    start_urls = ['https://news.163.com/']
    # url_index=[2,3,5,6]
    url_index = [2]
    url_models=[]

    def __init__(self):
        self.bro=webdriver.Chrome()
    def parse_model(self, response):
        div_list=response.xpath('/html/body/div/div[3]/div[4]/div[1]/div[1]/div/ul/li/div/div')
        for div in div_list:
            detail_url=div.xpath('./a/@href').extract_first()
            detail_title=div.xpath('.//h3/a/text()').extract_first()
            yield scrapy.Request(url=detail_url, callback=self.parse_detail,encoding='utf-8',meta={'detail_title':detail_title})



    def parse(self, response):
        li_list=response.xpath('//*[@id="index2016_wrap"]/div[1]/div[2]/div[2]/div[2]/div[2]/div/ul/li')
        for index in self.url_index:
            url=li_list[index].xpath('./a/@href').extract_first()
            news_type=li_list[index].xpath('./a/text()').extract_first()
            self.url_models.append(url)
        for url_model in self.url_models:
            yield scrapy.Request(url=url_model,callback=self.parse_model)
    def parse_detail(self,response):
        detail_title=response.meta['detail_title']
        detail_text=response.xpath('//*[@id="content"]/div[2]//text()').extract()
        detail_text=''.join(detail_text)
        item=WangyiproItem()
        item['detail_title']=detail_title
        item['detail_text']=detail_text
        yield item








